package edu.unc.ils.harvest;

import java.io.BufferedReader;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

// http://agricola.nal.usda.gov/help/z3950.html
// ISSN = 8
// TITLE = 4
public class AgricolaHarvester 
{
	
	public void getSerialsInfo() throws Exception
	{
		File serialsDir = new File("output" + File.separator + "NAL" + File.separator + "serials");
		File[] issns = serialsDir.listFiles();
		for (File issn: issns) 
		{
			if (issn.isDirectory())
			{

				File[] records = issn.listFiles();
				for (File record: records)
				{
		            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
		            DocumentBuilder db = dbf.newDocumentBuilder();
		            Document doc = db.parse(new FileInputStream(record));
				    XPathFactory  factory=XPathFactory.newInstance();
				    XPath xPath=factory.newXPath();
				    
				    
				    // 245 $a = title
				    NodeList nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='245']/subfield[@code='a']",
				            doc, XPathConstants.NODESET);
				   
				    String title = "";
				    for (int j=0; j<nodeList.getLength(); j++)
				    {
				            Node node = nodeList.item(j);
				            title = node.getNodeValue();
				    }
				    
				    // $650 $a
				    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='650']/subfield[@code='a']",
				            doc, XPathConstants.NODESET);
				   
				    List<String> subjects = new ArrayList<String>();;
				    for (int j=0; j<nodeList.getLength(); j++)
				    {
				            Node node = nodeList.item(j);
				            String subject = node.getNodeValue();
				            if (!subjects.contains(subject))
				            	subjects.add(subject);
				    }
				    
				    System.out.println(issn.getName() + "|" + title + "|" + subjects);
				}
			}
		}
	}
	public void getSerialsRecords()
	{
		try
		{
			Map<String, String> journals = getJournals();
			Z3950Client ah = new Z3950Client();
			// NAL catalog
			ah.connect("agricola.nal.usda.gov", 7090);

			for (String issn: journals.keySet())
			{

				String title = journals.get(issn);
				
				System.out.println(issn + ": " + title);
				ah.setOutputDirectory(new File("output" + File.separator + "NAL" + File.separator + "serials" + File.separator + issn));
				ah.search("@attrset bib-1 @attr 1=8 \"" + issn + "\"", title, issn);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public Map<String,String> getJournals() throws Exception
	{
		BufferedReader br = new BufferedReader(new FileReader("/Users/cwillis/dev/mrc/journal_analysis/open_access_agricola.txt"));
		String line;
		Map<String, String> journalMap = new HashMap<String, String>();
		while ((line = br.readLine()) != null) 
		{
			String[] fields = line.split("\\t");
			String title = fields[0];
			String issn = fields[1];
			journalMap.put(issn, title);
		}
		return journalMap;
	}
	public void getArticleRecords()
	{
		try
		{
			Map<String, String> journals = getJournals();
			Z3950Client ah = new Z3950Client();
			// Agricola Database
			ah.connect("agricola.nal.usda.gov", 7190);
			
			// Get all records that match the specified title.
			for (String issn: journals.keySet())
			{
				Thread.sleep(1000);
				String title = journals.get(issn);
				ah.setOutputDirectory(new File("output" + File.separator + "NAL" + File.separator + "articles" + File.separator + issn));
				ah.search("@attrset bib-1 @attr 1=1016 \"" + title + "\"", title, null);
			}
			
		} catch (Exception e) {
			e.printStackTrace();
		}
		
	}
	
	
	public void getArticleInfo() throws Exception
	{
		FileWriter documents = new FileWriter("output" + File.separator + "NAL" + File.separator + "agricola_articles.txt");
		FileWriter subjects = new FileWriter("output" + File.separator + "NAL" + File.separator + "agricola_subjects.txt");
		
		// ISSN, Title
		Map<String, String> journals = getJournals(); // new HashMap<String,String>();//
		//journals.put("1976-6696", "BMB reports");
		
		for (String issn: journals.keySet())
		{
			File issnDir = new File("output" + File.separator + "NAL" + 
				File.separator + "articles" + File.separator + issn);

			File[] records = issnDir.listFiles();
			if (records == null)
				continue;
			
			for (File record: records)
			{
	            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
	            DocumentBuilder db = dbf.newDocumentBuilder();
	            Document doc = db.parse(new FileInputStream(record));
			    XPathFactory  factory=XPathFactory.newInstance();
			    XPath xPath=factory.newXPath();
			    
			    NodeList nodeList = (NodeList)xPath.evaluate("/collection/record/controlfield[@tag='001']",
			            doc, XPathConstants.NODESET);
			    String id = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            id = node.getNodeValue();
			    }
			    
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='773']/subfield[@code='t']",
			            doc, XPathConstants.NODESET);
			    String journalTitle = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            journalTitle = node.getNodeValue();
			    }
			    
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='245']/subfield[@code='a']",
			            doc, XPathConstants.NODESET);
			    String articleTitle = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            articleTitle = node.getNodeValue();
			    }
			    
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='520']/subfield[@code='a']",
			            doc, XPathConstants.NODESET);
			    String abs = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            abs += node.getNodeValue();
			    }
			    
			    // Topical
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='650']/subfield[@code='a']",
			            doc, XPathConstants.NODESET);
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            String subject = node.getNodeValue();
		            subjects.write(id + "|" + j + "|" + subject + "|650\n" +
		            		"");
			    }
			    
			    // Geographic
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='651']/subfield[@code='a']",
			            doc, XPathConstants.NODESET);
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            String subject = node.getNodeValue();
		            subjects.write(id + "|" + j + "|" + subject + "|651\n");
			    }
			    // Uncontrolled
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='653']/subfield[@code='a']",
			            doc, XPathConstants.NODESET);
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            String subject = node.getNodeValue();
		            subjects.write(id + "|" + j + "|" + subject + "|653\n");
			    }
			    
			    // URL
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='856']/subfield[@code='u']",
			            doc, XPathConstants.NODESET);
			    String url = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            url = node.getNodeValue();
			    }
			    documents.write(id + "|" + journalTitle + "|" + articleTitle + "|" + url + "\n");
	
			    documents.flush();
			    subjects.flush();
			}


		}
		documents.close();
		subjects.close();
	}
	
	public static void main(String[] args) throws Exception {
		//AgricolaHarvester h = new AgricolaHarvester();
		//h.getSerialsInfo();
		//h.getArticleRecords();
		//h.getArticleInfo();
	}
}
